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-- file Scanner. Mesa 

-- last modified by Satterthwaite, June 16, 1978 2:56 PM 

DIRECTORY 

AltoDefs: FROM "altodefs" USING [CharsPerWord , maxword, PageSize], 
lODefs: FROM "iodefs" USING [CR, TAB, WriteChar, WriteNumber, WriteString] , 
LALRDefs: FROM "lalrdefs" 
USING [ 

LALRTable, Symbol, SymbolRecord, VocabHashEntry. 
endmarker, hashval , 

tokenARROW, tokenCHAR, tokenDOT, tokenDOTS, tokenEQUAL, 
tokenGE, tokenGREATER, tokenID, tokenLE, tokenLESS, 
tokenLNUM, tokenLSTR, tokenMINUS, tokenNUM, tokenSTR], 
LitDefs: FROM "litdefs" 

USING [FindLitDescriptor, FindLiteral, FindStringLiteral], 
PlDefs: FROM "pldefs", 
StreamDefs: FROM "streamdefs" 
USING [ 

StreamHandle, Streamlndex, 

Getlndex, Modifylndex, Normal izelndex, ReadBlock, Setlndex, 
StreamError] , 
StringDefs: FROM "stringdefs" USING [SubStringDescriptor , AppendString] , 
SymTabDefs: FROM "symtabdefs" USING [EnterString] , 
SystemDefs: FROM "systemdefs" 
USING [ 

AllocateHeapString , Al locatePages , FreeHeapString, FreePages, 
PruneHeap]; 

Scanner: PROGRAM 

IMPORTS lODefs, LitDefs, StreamDefs, StringDefs, SymTabDefs, SystemDefs 

EXPORTS PlDefs SHARES LALRDefs « 
BEGIN 
OPEN LALRDefs; 

dHashTab: DESCRIPTOR FOR ARRAY OF VocabHashEntry; 

dScanTab: DESCRIPTOR FOR ARRAY CHARACTER [40C..177C] OF Symbol; 

vocab: STRING; 

dVocablndex: DESCRIPTOR FOR ARRAY OF CARDINAL; 

NUL: CHARACTER = OC; 

CR: CHARACTER = lODefs.CR; 

ControlZ: CHARACTER = 32C; -- Bravo escape char 

stream: StreamDefs .StreamHandle; -- the input stream 
streamOrigin: StreamDefs .Streamlndex; 

TextPages: CARDINAL = 6; 

TextWords: CARDINAL = TextPages*Al toDefs . PageSize; 

TextChars: CARDINAL = TextWords*AltoDefs .CharsPerWord; 

tB: POINTER TO PACKED ARRAY [0 .. TextChars) OF CHARACTER; 
tl, tMax: [0. .TextChars]; 
tOrigin. tLimit: CARDINAL; 
tEnded: BOOLEAN; 

FillTextBuffer: PROCEDURE - 
BEGIN 

words: [0 . .TextWords] ; 
bytes: [0. .AltoDefs. CharsPerWord) ; 
tOrigin <- tLimit; 
IF tEnded 

THEN tMax ^ 
ELSE 

BEGIN 

words ♦- StreamDefs .ReadBlock[stream, tB, TextWords]; 

bytes <- StreamDefs .Getlndex[stream] . byte MOD Al toDefs .CharsPerWord; 

IF bytes ^ THEN words ♦- words-1; 

tMax ♦- words*AltoDefs .CharsPerWord + bytes; 

IF tMax < TextChars THEN tEnded <- TRUE; 

tLimit *- tOrigin + tMax; 

END; 
IF tMax - THEN BEGIN tB[0] <- NUL; tMax <~ 1 END; 
tl ^ 0; RETURN 
END; 
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buffer: STRING <- NIL; -- token assembly area 

iMax: CARDINAL; -- iMax ■ buff er.maxlength 

desc: StringDefs .SubStringDescriptor ; -- initial buffer segment 

nErrors: CARDINAL; -- lexical errors 

Buf ferOverflow: ERROR « CODE; 

ExpandBuffer: PROCEDURE - 
BEGIN 

oldBuffer: STRING ♦- buffer; 

IF OldBuffer. length > 2000 THEN ERROR BufferOverf low; 
buffer *- SystemDefs.AnocateHeapString[2*oldBuffer. length]; 
StringDefs.AppendString[buffer. oldBuffer]; 
iMax ♦- buffer, length ^ buff er.maxlength; 
SystemDefs.FreeHeapString[oldBuffer]; 
desc. base ^ buffer; 
RETURN 
END; 

char: CHARACTER; -- current (most recently scanned) character 

NextChar: PROCEDURE ■ -- also expanded inline within Atom 
BEGIN 

IF (tl^-tl+l) » tMax THEN Fi 1 ITextBuf f er[]; 
char ♦- tB[tI]; 
RETURN 
END; 

Atom: PUBLIC PROCEDURE RETURNS [symbol: SymbolRecord] « 
BEGIN 

OPEN symbol ; 
DO 

WHILE char IN [NUL.,' ] 
DO 

SELECT char FROM 
ControlZ -> 

UNTIL char - CR 
DO 
IF (tl<-tl+l) « tMax THEN 

BEGIN IF tEnded THEN GO TO EndFile; Fil ITextBuf fer[] END; 
char <- tB[tI]; 
ENDLOOP; 
ENDCASE; 
IF (tI^tI+1) = tMax THEN 

BEGIN IF tEnded THEN GO TO EndFile; Fil ITextBuf fer[] END; 
char <- tB[tI]; 



ENDLOOP; 


















index <- tOrigin + 


tl; 


value 


^ 0;- 












SELECT char FROM 


















'a, 'b. 'c, 'd. 


'e. 


'f. 'g. 


•h, 


'i. 


'J. 


'k. 


•1. 


•m 


'n, *o, 'p, 'q. 


'r. 


•s. 't. 


'u, 


•v. 


'w, 


'x. 


'y» 


'z 



BEGIN 

i: CARDINAL; 

i *- 0; 

DO 

buffer[i] ^ char; 

IF (tl*-tl+l) « tMax THEN Fil ITextBuf fer[]; 

char <r tB[tI]; 

SELECT char FROM 

IN C'a..'z], IN ['A..'Z]. IN [♦0..'9] «> 

IF (i «- i + 1) >« iMax THEN ExpandBuf f er[]; 
ENDCASE «> EXIT; 

ENDLOOP; 
desc. length *- i+1; 

class *- tokenID; value <- SymTabDefs .EnterString[@desc]; 
GO TO GotNext 
END; 

•A. 'B, 'C, 'D. 'E, 'F, 'G. 'H, 'I, 'J, 'K, 'L. 'M, 
•N, '0. 'P, *Q, 'R, 'S. 'T, 'U, W. 'W, 'X, 'Y, 'Z -> 

BEGIN 

first, last: CARDINAL; 
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uld: BOOLEAN; 
i, j, h: CARDINAL; 
si, s2: CARDINAL; 

i ^ 0; uld ^ TRUE; first ^ last ^ char-OC; 
DO 

buffer[i] *■ char; 

IF (tI^tI+1) - tMax THEN FilUextBufferC]; 
char ^ tB[tI]; 
SELECT char FROM 
IN C'A..'Z] »> 

BEGIN last <- char-OC; 

IF (i ^ i+1) >- iMax THEN ExpandBuff erC]; 
END; 
IN [•a..'z]. IN ['0..'9] »> 
BEGIN uld ^ FALSE; 

IF (i ^ i+1) >- iMax THEN ExpandBuff er[]; 
END; 
ENDCASE -> EXIT; 
ENDLOOP; 
i *- i+1; 
IF uld 
THEN 
BEGIN 

h ^ ((first*128-first) + last) MOD hashval + 1; 
WHILE (j <- dHashTab[h].symptr) ff 
DO 

IF dVocabIndex[j]-(s2<-dVocabIndex[j-l]) ■ i 
THEN 

FOR si IN [0 ., i) 
DO 

IF buffer[sl] # vocab[s2] THEN EXIT; 
s2 ♦- S2+1; 
REPEAT 

FINISHED => BEGIN class <- j ; 60 TO GotNext END; 
ENDLOOP; 
IF (h ^ dHashTab[h].link) « THEN EXIT; 
ENDLOOP; 
END; 
desc. length ♦- i; 

class ^ tokenID; value <- SymTabDefs .EnterString[@desc]; 
GO TO GotNext 
END; 

'0, '1. '2, '3, '4. '5, '6, '7, '8. '9 «> 
BEGIN 

V, vlO. v8: LONG INTEGER; 
scale: CARDINAL; 

valid, validlO, validS, octal: BOOLEAN; 
MaxLiteral: CARDINAL » Al toDef s .maxword; 

vRep: ARRAY [0 . .SIZE[LONG INTEGER]) OF WORD; -- machine dependent 
vlO <- v8 ♦- 0; validlO <- validS «- TRUE; 
WHILE char IN ['0. . '9] 
DO 

IF validlO THEN [vlO. validlO] ^ AppendDigitlO[vlO , char]; 
IF valid8 THEN [v8, validS] <- AppendDigit8[v8, char]; 
NextChar[]; 
ENDLOOP; 
SELECT char FROM 
•B. 'C «> 
BEGIN 

class <r IF char - 'C THEN tokenCHAR ELSE tokenNUM; 
V ^ v8; valid *■ validS; octal ♦- TRUE; 
END; 
ENDCASE => 
BEGIN 

class ^ tokenNUM; v *- vlO; valid <- validlO; octal <- FALSE; 
END; 
SELECT char FROM 
•B, 'C, 'D «> 
BEGIN 

NextChar[]; 
IF class " tokenNUM 
THEN 

BEGIN scale ^ 0; 
WHILE char IN ['0. . '9] 
DO 
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scale *■ 10*scale + CARDINAL[char-'0]; 
NextChar[]; 
ENDLOOP; 
THROUGH [1 .. scale] WHILE valid 
DO 
IF octal 

THEN [v, valid] ^ AppendDigit8[v, '0] 
ELSE [v, valid] ^ AppendDigitlO[v, '0]; 
ENDLOOP; 
END; 
END; 
ENDCASE; 
vRep <- LOOPHOLE[v]; 
IF V <» MaxLiteral 

THEN value ♦■ LitDef s.FindLiteral[vRep[0]] 
ELSE 
BEGIN 

IF class » tokenCHAR THEN valid ^ FALSE; 
class ^ tokenLNUM; 

value ^ LitDefs.FindLitDescriptor[DESCRIPTOR[vRep]]; 
END; 
IF -valid THEN 
BEGIN 
nErrors <- nErrors + 1; ScanError[number, index]; 



END; 
GO TO 
END; 


GotNext 












BEGIN 
class 
END; 


' : . '♦-, 
•[. ']. 

<- dScan 


Tab[char] 




GO TO 


GetNext 



BEGIN 

NextChar[]; 

class <- tokenCHAR; value <- LitDef s. FindLiteral[char-OC] ; 

GO TO GetNext 

END; 

" -> 

BEGIN 

i: CARDINAL; 

i *• 0; 

DO 

IF (tI^tI+1) « tMax THEN 

BEGIN IF tEnded THEN GO TO EOFEnd; Fil ITextBuff er[] END; 

char ^ tB[tI]; 

SELECT char FROM 

BEGIN 

IF (tKtI+1) = tMax THEN Fil ITextBuf fer[]; 
char ^ tB[tI]; 

IF char # '" THEN GO TO QuoteEnd; 
END; 
ENDCASE; 
IF i >== iMax 

THEN ExpandBuffer[ 
IBufferOverf low «> 
BEGIN 

nErrors ♦- nErrors + 1; 
ScanError[string, tOrigin+tl]; 
i <- 0; 
CONTINUE 
END]; 
buffer[i] ^ char; i <- i+1; 
REPEAT 

QuoteEnd -> NULL; 

EOFEnd -> BEGIN Fil ITextBuff er[] ; char ^ tBCtI] END; 
ENDLOOP; 
desc. length <- i ; 

value ♦- LitDefs . FindStringLiteral[0desc]; 
IF char « 'L 

THEN BEGIN class <- tokenLSTR; GO TO GetNext END 
ELSE BEGIN class ^ tokenSTR; GO TO GotNext END 
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END; 

- -> 

BEGIN 

pChar: CHARACTER; 

NextChar[]; 

IF char # '- 

THEN BEGIN class ^ tokenMINUS; GO TO GotNext END; 
char ^ NUL; 
DO 

pChar ♦• char; 
IF (tI^tI+1) « tMax THEN 

BEGIN IF tEnded THEN GO TO EndFile; FinTextBuffer[] END; 
char ♦- tB[tI]; 
SELECT char FROM 

•- ■> IF pChar - •- THEN EXIT; 
CR -> EXIT; 
ENDCASE; 
ENDLOOP; 
NextChar[]; 
END; 



. «> 

BEGIN 

NextChar[]; 

IF char ■ ' . 
THEN BEGIN 
ELSE BEGIN 

END; 



class ♦- tokenDOTS; 
class <- tokenDOT; 



GO TO GetNext 
GO TO GotNext 



END 
END 



- «> 

BEGIN 

NextChar[]; 

IF char « '> 
THEN BEGIN 
ELSE BEGIN 

END; 



class ♦- tokenARROW; 
class ♦- tokenEQUAL; 



GO TO GetNext 
GO TO GotNext 



END 
END 



BEGIN 

NextChar[]; 

IF char » *■ 
THEN BEGIN 
ELSE BEGIN 

END; 



class 
class 



tokenLE; 
tokenLESS; 



GO TO GetNext 
GO TO GotNext 



END 
END 



BEGIN 

NextChar[]; 

IF char » •" 
THEN BEGIN 
ELSE BEGIN 

END; 



class ♦- tokenGE; 
class *- tokenGREATER; 



GO TO GetNext 
GO TO GotNext 



END 
END 



ENDCASE »> 
BEGIN 

class <- dScanTab[char]; 
IF class # THEN GO TO GetNext; 
NextChar[]; 

nErrors <- nErrors + 1; ScanError[char» index]; 
END; 



REPEAT 

GetNext => 
BEGIN 

IF (tl^-tl + l) « tMax THEN FillTextBuf fer[] ; 
char *- tB[tI]; 



END; 
GotNext 


«> 


NULL; 


EndFile 


-> 




BEGIN 






FillTextBi 


jfferCl; 


class 


♦- endmarker 


END; 






ENDLOOP; 






RETURN 







char <r. tB[tI]; 
index ♦- tOrigin; 



value ^ 0; 
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END; 

"- numerical conversion 

Digit: ARRAY CHARACTER [•0..'9] OF CARDINAL - [0,1,2,3,4,5,6,7.8,9]; 

AppendDigitlO: PROCEDURE [v: LONG INTEGER, digit: CHARACTER [•C.'g]] 
RETURNS [newV: LONG INTEGER, valid: BOOLEAN] » 
BEGIN 

MaxV: LONG INTEGER - 214748364; -- (2**31-1)/10 

MaxD: CARDINAL « 7; -- (2**31-1) MOD 10 

d: [0..9] ■ Digit[digit]; 

valid *■ V < MaxV OR (v - MaxV AND d <« MaxD); 
newV <- 10*v + d; 
RETURN 
END; 

AppendDigitS: PROCEDURE [v: LONG INTEGER, digit: CHARACTER ['0..'9]] 
RETURNS [newV: LONG INTEGER, valid: BOOLEAN] « 
BEGIN 

MaxV: LONG INTEGER « 1777777777B; -- {2**31-l)/8 

MaxD: CARDINAL = 7B; -- {2**31-1) MOD 8 

d: [0..9] - Digit[digit]; 

valid ♦- (d < 8) AND (v < MaxV OR (v = MaxV AND d <« MaxD)); 
newV <- 8*v + d; 
RETURN 
END; 

-- initial ization/final ization 

Scanlnit: PUBLIC PROCEDURE [ 

textStream: StreamDef s.StreamHandle, tablePtr: POINTER TO LALRTable] « 
BEGIN 

BEGIN OPEN tablePtr. scantable; 

dHashTab ^ DESCRIPTOR [hashtab]; 

dScanTab <- DESCRIPTOR [scantab]; 

vocab ♦- LOOPHOLE[@vocabbody, STRING]; 

dVocablndex <- DESCRIPTOR [vocabindex]; 

END; 
IF buffer « NIL THEN buffer *- SystemDef s . Al locateHeapString[256] ; 
iMax ♦" buffer, length <- buff er .maxlength; 
desc.base *- buffer; desc. offset <r O; 
stream <- textStream; 

streamOrigin <- StreamDef s .Getlndex[stream]; 
tB <- SystemDef s.AnocatePages[T€xtPages]; 
tOrigin ♦- tLimit ♦- 0; tMax ♦- 0; tEnded ♦- FALSE; 
FillTextBufferC]; char ^ tB[tI]; 
nErrors ♦- 0; 
RETURN 
END; 

ScanReset: PUBLIC PROCEDURE RETURNS [CARDINAL] « 
BEGIN 

SystemDef s .FreePages[tB]; 
IF buffer ^ NIL 

THEN BEGIN SystemDef s . FreeHeapString[buf fer] ; buffer <- NIL END; 
[] ♦- SystemDef s.PruneHeap[] ; 
RETURN [nErrors] 
END; 

-- error handling 

Streamlndex: TYPE = StreamDef s .Streamlndex; 

PrintTextLine: PROCEDURE [origin: Streamlndex] RETURNS [start: Streamlndex] « 
BEGIN OPEN lODefs; 
linelndex: Streamlndex; 
char: CHARACTER; 
n: [1..100]; 

start <- linelndex <- origin; 
FOR n IN [1..100] UNTIL linelndex « [0, 0] 

DO 

linelndex <- StreamDefs.ModifyIndex[l inelndex, ~1]; 
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StreamDef s.SetIndex[stream, linelndex]; 

IF stream. get[stream] - CR THEN EXIT; 

start ^ 1 inelndex; 

ENDLOOP; 
StreamDef s. Set I ndex[st ream, start]; 
FOR n IN [1..100] 

DO 

char ^ stream. get[stream I StreamDef s.StreamError «> EXIT]; 

SELECT char FROM 

CR, ControlZ ■> EXIT; 
ENDCASE »> WriteChar[char]; 

ENDLOOP; 
WriteChar[CR]; RETURN 
END; 

ResetScanlndex: PUBLIC PROCEDURE [index: CARDINAL] « 
BEGIN 

page: CARDINAL; 

IF index -IN [tOrigin .. tLimit) 
THEN 

BEGIN 

page <- index/(AHoDef s.PageSize*Al toDef s.CharsPerWord) ; 

tOrigin ^ tLimit <- page*(Al toDef s .PageSize*Al toDef s.CharsPerWord) ; 

tMax <- 0; 

StreamDef s .Setlndex[stream, 

[page: streamOrigin.page+page, byte: streamOrigin.byte]]; 

FinTextBuffer[]; 

END; 
tl ^ index - tOrigin; char <- tB[tI]; RETURN 
END; 

ScanError: PROCEDURE [code: {number, string, char}, tokenlndex: CARDINAL] - 
BEGIN 
ErrorContext[ 

SELECT code FROM 

number => "Invalid Number"L. 
string => "String Too Long"L, 
char => "Invalid Character"L, 
ENDCASE => NIL, 
tokenlndex]; 
RETURN 
END; 

ErrorContext: PUBLIC PROCEDURE [message: STRING, tokenlndex: CARDINAL] » 
BEGIN OPEN lODefs; 

savelndex: Streamlndex = StreamDefs.GetIndex[stream]; 
origin: Streamlndex = StreamDef s .Normal izelndex[ 

[page: streamOrigin.page, byte: streamOrigin.byte+tokenlndex]]; 
char: CHARACTER; 
WriteChar[CR]; 

StreamDef s. Set I ndex[ stream, Print Tex tLine[origin]]; 
UNTIL StreamDefs,GetIndex[stream] « origin 

DO 

char <- stream. get[stream 1 StreamDef s .StreamError => EXIT]; 

WriteChar[IF char - TAB THEN TAB ELSE ' ]; 

ENDLOOP; 
WriteString["t "L]; WriteString[message] ; WriteString[" ["L]; 
WriteNumber[tokenIndex, [base:8, zerof ill :FALSE, unsigned:TRUE , columns:0]]; 
WriteChar[']]; WriteChar[CR]; 
StreamDefs.SetIndex[stream, savelndex]; RETURN 
END; 



END. 



